# waveplugs/rss.py
#
#

"""
    the rss mantra is of the following:

    -!- add a url with rss-add
    -!- use rss-scan <feed> to see what tokens you can use
    -!- use rss-additem <feed> to add the tokens
    -!- use rss-get <feed> to test the rss results
    -!- use rss-peek <feed> to poll the rss feed
    -!- for now its not possible to have the results polled into the wave, because
        the API can only react to events instead of creating them.

 """

## gozerlib imports

from gozerlib.persist.persist import Persist, PlugPersist
from gozerlib.utils.url import geturl2, striphtml, useragent
from gozerlib.utils.exception import handle_exception
from gozerlib.utils.generic import strippedtxt, fromenc, jsonstring
from gozerlib.utils.rsslist import rsslist
from gozerlib.utils.lazydict import LazyDict
from gozerlib.utils.statdict import StatDict
from gozerlib.utils.timeutils import strtotime
from gozerlib.commands import cmnds
from gozerlib.examples import examples
from gozerlib.utils.dol import Dol
from gozerlib.persist.pdod import Pdod
from gozerlib.persist.pdol import Pdol
from gozerlib.aliases import aliasset
from gozerlib.tests import tests
from gozerlib.users import GozerUsers
from gozerlib.botbase import BotBase

## google imports

from google.appengine.ext import db
from google.appengine.ext import deferred
from google.appengine.api.memcache import get, set, delete

import google

## json import

from simplejson import loads

## tinyurl import

try:
    from waveplugs.tinyurl import get_tinyurl
except:
    def get_tinyurl(url):
        return url

## basic imports

import feedparser
import time
import os
import types
import thread
import socket
import xml
import logging
import datetime

## MODEL

class RSSDbItem(db.Expando):

    feedurl = db.StringProperty(required=True)
    updated = db.DateTimeProperty(required=True)
    link = db.StringProperty()
    summary = db.StringProperty()
    author = db.StringProperty()

savelist = []
possiblemarkup = {'separator': 'set this to desired item separator', \
'all-lines': "set this to 1 if you don't want items to be aggregated", \
'tinyurl': "set this to 1 when you want to use tinyurls", 'skipmerge': \
"set this to 1 if you want to skip merge commits", 'reverse-order': 'set \
this to 1 if you want the rss items displayed with oldest item first'}

def txtindicts(result, d):

    """ return lowlevel values in (nested) dicts. """

    for j in d.values():

        if type(j) == types.DictType:
            txtindicts(result, j) 
        else:
            result.append(j)

def checkfordate(data, date):

    """ see if date is in data (list of feed items). """

    if not data:
        return False

    for item in data:

        try:
            d = item['updated']
        except (KeyError, TypeError):
            continue

        if date == d:
            return True

    return False

class RssException(Exception):
    pass

class Rss301(RssException):
    pass

class RssStatus(RssException):
    pass

class RssBozoException(RssException):
    pass

class RssNoSuchItem(RssException):
    pass

class Rssitem(Persist):

    """ item that contains rss data """


    def __init__(self, name="nonameset", url="", owner="noownerset", itemslist=['title', ], watchchannels=[], \
sleeptime=30*60, running=0):

        if True:
            filebase = 'gozerstore' + os.sep + 'plugs' + os.sep + 'waveplugs.rss' + os.sep + name
            Persist.__init__(self, filebase + os.sep + 'rss-core')

            if not self.data:
                 self.data = {}

            self.data = LazyDict(self.data)
            self.data['name'] = self.data.name or str(name)
            self.data['url'] = self.data.url or str(url)
            self.data['owner'] = self.data.owner or str(owner)
            self.data['itemslist'] = self.data.itemslist or list(itemslist)
            self.data['watchchannels'] = self.data.watchchannels or list(watchchannels)
            self.data['sleeptime'] = self.data.sleeptime or int(sleeptime)
            self.data['running'] = self.data.running or running
            self.lastpeek = Persist(filebase + os.sep + 'rss-lastpeek')
            
    def boot(self, name="", url="", itemslist=[], watchchannels=[], \
sleeptime=30*60, running=0, item={}):

        if True:
            self['name'] = name
            self['url'] = url
            self['itemslist'] = list(itemslist)
            self['watchchannels'] = list(watchchannels)
            self['sleeptime'] = int(sleeptime)
            self['running'] = int(running)
            self['stoprunning'] = 0
            self['botname'] = None

    def ownercheck(self, userhost):

        """ check is userhost is the owner of the feed. """

        try:
            return self.data.owner == userhost
        except KeyError:
            pass

        return False

    def save(self):

        """ save rss data. """

        Persist.save(self)
        self.lastpeek.save()

    def getdata(self):
        url = self.data['url']
        result = get(url)

        if result == None:
            result = self.fetchdata()
            set(url, result, self.data.sleeptime)
            logging.debug("rss - got result from %s" % url)
        else:
            logging.debug("rss - got result from %s *cached*" % url)

        return result

    def fetchdata(self):

        """ get data of rss feed. """

        url = self.data['url']
        result = feedparser.parse(url,agent=useragent())
        logging.debug("rss - fetch - got result from %s" % url)
        
        if result and result.has_key('bozo_exception'):
            logging.warn('rss - %s bozo_exception: %s' % (url, result['bozo_exception']))

        try:
            status = result.status
            logging.info("rss - status is %s" % status)
        except AttributeError:
            status = 200

        if status != 200 and status != 301 and status != 302:
            raise RssStatus(status)

        return result.entries

    def sync(self):

        """ refresh cached data of a feed. """

        result = self.fetchdata()
        set(self.data.url, result)

        if result:

            for item in result:
                self.additem(item) 

        return self

    def additem(self, item):

        """ add a item to the database. """

        try:
            try:
                up = item['updated']
            except KeyError:
                try:
                    up = item['published']
                except KeyError:
                    logging.error("rss - no updated or published field found in %s feed: %s" % (self.data.name, str(item)))
                    return

            dt = feedparser._parse_date(up)
            dtt = datetime.datetime.fromtimestamp(time.mktime(dt))
            if not dtt:
                logging.error("rss - %s feed .. can't determine time out of string %s" % (self.data.name, up))
                return

            if not self.got(dtt):
                logging.info("rss - %s feed .. inserting rss item with update time %s" % (self.data.name, dtt)) 
                dbitem = RSSDbItem(feedurl=self.data.url,updated=dtt)

                for i, value in item.iteritems():

                    if i == 'updated':
                        continue                            

                    logging.warn("%s %s %s" % (i, type(value), value))
                    setattr(dbitem, i, str(value))

                dbitem.put()
            else:
                logging.debug("rss - %s feed .. rss item with update time %s already there" % (self.data.name, dtt)) 

        except KeyError:
            logging.error("can't determine update time from %s feed" % self.data.name)

    def get(self, event):

        """ get items for user originating the event since lastpeeked. """

        users = GozerUsers()
        name = users.getname(event.userhost)

        try:
            lastpeeked = float(self.lastpeek.data[name])
        except (KeyError, ValueError):
            lastpeeked = 0
            logging.warn("last peek of %s is initialised" % name)

        logging.warn("using lastpeeked: %s for user %s" % (time.ctime(lastpeeked), name))
        result = db.GqlQuery("SELECT * FROM RSSDbItem WHERE feedurl = :1 AND updated > :2 ORDER BY updated DESC", self.data.url, datetime.datetime.fromtimestamp(lastpeeked)).fetch(100)

        logging.warn("getting rss items for %s (%s)" % (name, len(result)))

        if result:
            r = datetime.datetime.min
            for res in result:
                if res.updated:
                    if res.updated > r:
                        r = res.updated
                        self.lastpeek.data[name] = time.mktime(r.timetuple())
            logging.debug('lastpeek of %s set to %s' % (name, self.lastpeek.data[name]))
            self.lastpeek.save()
            return result
        else: 
            return []

    def got(self, updated):

        """ see if item exists. based on feedurl and updated time."""

        result = db.GqlQuery("SELECT * FROM RSSDbItem WHERE feedurl = :1 AND updated = :2", self.data.url, updated).get()

        if result:
            return True
        else: 
            return False

    def latest(self, nritems=1):

        result = db.GqlQuery("SELECT * FROM RSSDbItem WHERE feedurl = :1 ORDER BY updated DESC", self.data.feedurl, nritems).fetch(nritems)
        return result


    def all(self):

        """ return all items (tag data) from this feed. """

        result = db.GqlQuery("SELECT * FROM RSSDbItem WHERE feedurl = :1", self.data.url).fetch(1000)
        return result

    def search(self, item, search):
        res = []

        for result in self.all():
            try:
                title = result['title']
                txt = result[item]
            except KeyError:
                continue

            if search in title.lower():
                if txt:
                   res.append(txt)

        return res


class Rssdict(PlugPersist):

    """ dict of rss entries """

    def __init__(self, filename):
        self.feeds = {}
        PlugPersist.__init__(self, filename)

        if not self.data:
            self.data = {}
            self.data['names'] = []
        else:
            if not self.data.has_key('names'):
                self.data['names'] = []
            for name in self.data['names']:
                self.feeds[name] = Rssitem(name)

        self.itemslists = Pdol('gozerstore' + os.sep + 'plugs' + os.sep + 'waveplugs.rss' + os.sep + filename + '.itemslists')
        self.markup = Pdod('gozerstore' + os.sep + 'plugs' + os.sep + 'waveplugs.rss' + os.sep + filename + '.markup')
        self.lastpeek = Pdod('gozerstore' + os.sep + 'plugs' + os.sep + 'waveplugs.rss' + os.sep + filename + '.lastpeek')

    def save(self, namein=None):

        """ save all feeds or provide a feedname to save. """

        Persist.save(self)

        for name, rss in self.feeds.iteritems():

            if namein and name != namein:
                continue

            try:
                rss.save()
            except Exception, ex:
                handle_exception()

    def size(self):

        """ return number of rss feeds. """

        return len(self.feeds)

    def add(self, name, url, owner):

        """ add rss item. """

        logging.info('rss - adding %s - <%s> - (%s)' % (name, url, owner))

        if name not in self.data['names']:
            self.data['names'].append(name)

        self.feeds[name] = Rssitem(name, url, owner, ['title', ])
        self.save(name)

    def delete(self, name):

        """ delete rss item by name. """

        target = None

        for j, i in self.feeds.iteritems():

           if i.name == name:
               target = i

        if target:

            try:
                target.data.running = 0
                del self.feeds[name]
                self.save()
            except:
                pass

    def byname(self, name):

        """ return rss item by name. """

        try:
            return self.feeds[name]
        except KeyError:
            return

    def getdata(self, name):

        """ get data of rss feed. """

        rssitem = self.byname(name)

        if rssitem == None:
            raise RssNoSuchItem("no %s rss item found" % name)

        return rssitem.getdata()

class Rsswatcher(Rssdict):

    """ rss watchers. """ 

    def get(self, name, event):
        return self.feeds[name].get(event)

    def sync(self, name):

        """ sync a feed. """

        feed = self.byname(name)

        if feed:
            feed.sync()

    def ownercheck(self, name, userhost):

        """ check if userhost is the owner of feed. """

        try:
            return self.feeds[name].ownercheck(userhost)
        except KeyError:
            pass

        return False

    def changeinterval(self, name, interval):

        """ not implemented yet. """

        pass

    def stopwatchers(self):

        """ stop all watcher threads. """

        for j, z in self.data.iteritems():

            if z.data.running:
                z.data.stoprunning = 1

    def dowatch(self, name, sleeptime=1800):

        """ start a watcher. """

        rssitem = self.byname(name)

        if not rssitem == None:
            logging.warn("rss - no %s rss item available" % name)
            return

        while 1:

            try:
                self.watch(name)
            except Exception, ex:
                logging.warn('rss - %s feed error: %s' % (name, str(ex)))
                if not rssitem.data.running:
                    break
            else:
                break

    def makeresult(self, name, target, data):

        """ make a result (txt) of a feed depending on its itemlist. """

        res = []

        for j in data:
            tmp = {}

            if not self.itemslists.data[jsonstring([name, target])]:
                return []

            for i in self.itemslists.data[jsonstring([name, target])]:
                try:
                    tmp[i] = unicode(j[i])
                except KeyError:
                    continue

            res.append(tmp)

        return res

    def watch(self, name):

        """ start a watcher thread """

        # get basic data
        logging.debug('trying %s rss feed watcher' % name)
        rssitem = self.feeds[name]

        if rssitem == None:
            raise RssNoItem()

        rssitem.data.running = 1
        rssitem.data.stoprunning = 0
        rssitem.getdata()
        rssitem.save()
        logging.info('rss - started %s rss watch' % name)

    def makeresponse(self, name, res, channel, sep=" .. "):

        """ loop over result to make a response. """

        result = u""

        try:
            itemslist = self.itemslists.data[jsonstring([name, channel])]
        except KeyError:
            rssitem = self.byname(name)

            if rssitem == None:
                return "no %s rss item" % name
            else:
                self.itemslists.data[jsonstring([name, channel])] = rssitem.data.itemslist
                self.itemslists.save()

        for j in res:

            if self.markup.get(jsonstring([name, channel]), 'skipmerge') and 'Merge branch' in j['title']:
                continue

            resultstr = u""

            for i in self.itemslists.data[jsonstring([name, channel])]:
                try:
                    item = getattr(j, i)

                    if not item:
                        continue

                    if item.startswith('http://'):

                        if self.markup.get(jsonstring([name, channel]), 'tinyurl'):

                            try:
                                tinyurl = get_tinyurl(item)

                                if not tinyurl:
                                    resultstr += u"<%s> - " % item
                                else:
                                    resultstr += u"<%s> - " % tinyurl[0]

                            except Exception, ex:
                                handle_exception()
                                resultstr += u"<%s> - " % item
	
                        else:
                            resultstr += u"<%s> - " % item

                    else:
                        resultstr += u"%s - " % item.strip()

                except KeyError:
                    continue

            resultstr = resultstr[:-3]

            if resultstr:
                result += u"%s %s " % (resultstr, sep)

        return result[:-(len(sep)+2)]

    def peek(self, name, event=None, *args):

        """ poll a feed. display not yet shown items. """

        rssitem = self.byname(name)

        if not rssitem:
            logging.debug("rss - skipping peek of %s" % name)
            return

        try:
            if event:
                users = GozerUsers()
                user = users.getname(event.userhost)

            res2 = self.get(name, event)

            if not res2:
                logging.info("no updates for %s feed available" % rssitem.data.name)
                return False

            if event:
                loopover = [event.userhost, ]
            else:
                loopover = rssitem.data.watchchannels 

            for item in loopover:

                if event:
                    channel = item
                else:

                    try:
                        (botname, channel) = item
                    except:

                        try:
                            (botname, channel) = loads(item)
                        except:
                            logging.warn('rss - %s is not in the format \
(botname,channel)' % str(item))

                bot = BotBase('rss')

                if self.markup.get(jsonstring([name, channel]), 'reverse-order'):
                    res2 = res2[::-1]

                if self.markup.get(jsonstring([name, channel]), 'all-lines'):

                    for i in res2:
                        response = self.makeresponse(name, [i, ], channel)

                        if event:
                            event.reply(response)
                        else:

                            try:
                                bot.say(channel, response)
                            except Exception, ex:
                                handle_exception()

                else:
                    sep =  self.markup.get(jsonstring([name, channel]), 'separator')

                    if sep:
                        response = self.makeresponse(name, res2, channel, sep=sep)
                    else:
                        response = self.makeresponse(name, res2, channel)

                    if event:
                        event.reply(response)
                    else:

                        try:
                            bot.say(channel, "%s: %s" % (rssitem.data.name, response))
                        except Exception, ex:
                            handle_exception()

        except Exception, ex:
            handle_exception(txt=name)

        return True

    def stopwatch(self, name):

        """ stop watcher thread. """

        try:
            self.feeds[name].data.running = 0
            self.feeds[name].save()
            return True

        except KeyError:
            return False

    def list(self):

        """ return of rss names. """

        feeds = self.feeds.keys()
        return feeds

    def runners(self):	

        """ show names/channels of running watchers. """

        result = []

        for j, z in self.feeds.iteritems():

            if z.data.running == 1 and not z.data.stoprunning: 
                result.append((z.data.name, z.data.watchchannels))

        return result

    def getfeeds(self, botname, channel):

        """ show names/channels of running watcher. """

        result = []

        for j, z in self.feeds.iteritems():

            if jsonstring([botname, channel]) in z.data.watchchannels or [botname, channel] in z.data.watchchannels:
                result.append(z.data.name)

        return result

    def url(self, name):

        """ return url of rssitem. """

        return self.feeds[name].data.url

    def seturl(self, name, url):

        """ set url of rssitem. """

        self.feeds[name].data.url = url
        self.feeds[name].save()
        return True

    def scan(self, name):

        """ scan a rss url for tokens. """

        try:
            result = self.getdata(name)
        except RssException, ex:
            logging.warn('rss - %s error: %s' % (name, str(ex)))
            return

        if not result:
            return

        keys = []

        items = self.feeds[name].fetchdata()

        for item in items:

            for key in item:
                keys.append(key)            

        statdict = StatDict()

        for key in keys:
            statdict.upitem(key)

        return statdict.top()  

    def search(self, name, item, search):

        """ search titles of a feeds cached data. """

        return self.feeds[name].search(item, search)


    def searchall(self, item, search):

        """ search titles of all cached data. """

        res = []

        for name, feed in self.feeds.iteritems():

            res.append(str(feed.search(item, search)))

        return res

    def all(self, name, item):

        """ search all cached data of a feed. """

        res = []

        for result in self.feeds[name].all():

            try:
                txt = getattr(result, item)
            except AttributeError:
                continue

            if txt:
                res.append(txt)

        return res

    def peekall(self, event=None):

        """ peek all feeds. """

        for j, z in self.feeds.iteritems():

            if z.data.running:
                delete(z.data.url)
                self.peek(z.data.name, event)

    def periodical(self):

        """ run periodical .. refresh items. """

        for j, z in self.feeds.iteritems():
            logging.warn('deleting %s cache entry' % j)
            self.sync(z.data.name)


    def startwatchers(self):

        """ start watcher threads """

        for j, z in self.feeds.iteritems():
            if z.data.running:
                self.watch(z.data.name)


def periodical():

    """ function thats called by periodical. """

    logging.debug("rss - running periodical")
    deferred.defer(doperiodical)

def doperiodical():
    watcher.periodical()


# the watcher object
watcher = Rsswatcher('rss')

assert(watcher)

def size():

    """ return number of watched rss entries. """

    return watcher.size()

def save():

    """ save watcher data. """

    watcher.save()
    
def handle_rssadd(bot, ievent):

    """ rss-add <name> <url> .. add a rss item. """

    try:
        (name, url) = ievent.args
    except ValueError:
        ievent.missing('<name> <url>')
        return

    watcher.add(name, url, ievent.userhost)
    ievent.reply('rss item added')

cmnds.add('rss-add', handle_rssadd, 'USER')
examples.add('rss-add', 'rss-add <name> <url> to the rsswatcher', 'rss-add \
gozerbot http://core.gozerbot.org/hg/dev/0.9/rss-log')
tests.add('rss-add dev http://core.gozerbot.org/hg/dev/0.9/rss-log')

def handle_rssdel(bot, ievent):

    """ rss-del <name> .. delete a rss item. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    if watcher.byname(name):
        if not watcher.ownercheck(name, ievent.userhost):
            ievent.reply("you are not the owner of the %s feed" % name)
            return
        watcher.stopwatch(name)
        watcher.delete(name)
        ievent.reply('rss item deleted')
    else:
        ievent.reply('there is no %s rss item' % name)

cmnds.add('rss-del', handle_rssdel, ['USER', ])
examples.add('rss-del', 'rss-del <name> .. remove <name> from the rsswatcher', 'rss-del mekker')
tests.add('rss-del mekker')

def handle_rsssync(bot, ievent):

    """ rss-del <name> .. delete a rss item. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    if name in watcher.feeds:
        watcher.feeds[name].sync()
        ievent.done()
    else:
        ievent.reply("no %s feed available" % name)

cmnds.add('rss-sync', handle_rsssync, ['USER', ])
examples.add('rss-sync', 'rss-sync <name> .. sync <name> feed', 'rss-sync mekker')
tests.add('rss-sync mekker')

def handle_rsswatch(bot, ievent):

    """ rss-watch <name> .. start watcher thread. """

    if not ievent.channel:
        ievent.reply('no channel provided')

    try:
        name, sleepsec = ievent.args
    except ValueError:

        try:
            name = ievent.args[0]
            sleepsec = 1800
        except IndexError:
            ievent.missing('<name> [secondstosleep]')
            return

    try:
        sleepsec = int(sleepsec)
    except ValueError:
        ievent.reply("time to sleep needs to be in seconds")
        return

    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss object" % name)
        return

    got = None

    if not rssitem.data.running or rssitem.data.stoprunning:

        if not rssitem.data.sleeptime:
            rssitem.data.sleeptime = sleepsec

        rssitem.data.running = 1
        rssitem.data.stoprunning = 0
        got = True
        watcher.save(name)

        try:
            watcher.watch(name)
        except Exception, ex:
            ievent.reply(str(ex))
            return

    if got:
        watcher.save()
        ievent.reply('watcher started')
    else:
        ievent.reply('already watching %s' % name)

cmnds.add('rss-watch', handle_rsswatch, 'USER')
examples.add('rss-watch', 'rss-watch <name> [seconds to sleep] .. go \
watching <name>', '1) rss-watch gozerbot 2) rss-watch gozerbot 600')
tests.add('rss-watch')

def handle_rssstart(bot, ievent):

    """ rss-start <name> .. start a rss feed to a user. """

    if not ievent.rest:
       ievent.missing('<feed name>')
       return

    name = ievent.rest
    rssitem = watcher.byname(name)
    target = ievent.channel

    if rssitem == None:
        ievent.reply("we don't have a %s rss object" % name)
        return

    if not rssitem.data.running:
        ievent.reply('%s watcher is not running' % name)
        return

    if jsonstring([bot.name, target]) in rssitem.data.watchchannels or [bot.name, target] in rssitem.data.watchchannels:
        ievent.reply('we are already monitoring %s on (%s,%s)' % (name, bot.name, target))
        return

    rssitem.data.watchchannels.append([bot.name, target])

    for item in rssitem.data.itemslist:
        watcher.itemslists.adduniq(jsonstring([name, target]), item)

    watcher.save()
    ievent.reply('%s started' % name)

cmnds.add('rss-start', handle_rssstart, ['RSS', 'USER'])
examples.add('rss-start', 'rss-start <name> .. start a rss feed \
(per user/channel) ', 'rss-start gozerbot')
tests.add('rss-start dev')

def handle_rssstop(bot, ievent):

    """ rss-start <name> .. start a rss feed to a user. """

    if not ievent.rest:
       ievent.missing('<feed name>')
       return

    name = ievent.rest
    rssitem = watcher.byname(name)
    target = ievent.channel

    if rssitem == None:
        ievent.reply("we don't have a %s rss feed" % name)
        return

    if not rssitem.data.running:
        ievent.reply('%s watcher is not running' % name)
        return

    try:
        rssitem.data.watchchannels.remove([bot.name, target])
    except ValueError:

        try:
            rssitem.data.watchchannels.remove([bot.name, target])
        except ValueError:
            ievent.reply('we are not monitoring %s on (%s,%s)' % (name, bot.name, target))
            return

    watcher.save()
    ievent.reply('%s stopped' % name)

cmnds.add('rss-stop', handle_rssstop, ['RSS', 'USER'])
examples.add('rss-stop', 'rss-stop <name> .. stop a rss feed \
(per user/channel) ', 'rss-stop gozerbot')
tests.add('rss-stop mekker')

def handle_rsschannels(bot, ievent):

    """ rss-channels <name> .. show channels of rss feed. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing("<name>") 
        return

    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss object" % name)
        return

    if not rssitem.data.watchchannels:
        ievent.reply('%s is not in watch mode' % name)
        return

    result = []

    for i in rssitem.data.watchchannels:
        result.append(str(i))

    ievent.reply("channels of %s: " % name, result, dot=True)

cmnds.add('rss-channels', handle_rsschannels, ['OPER', ])
examples.add('rss-channels', 'rss-channels <name> .. show channels', 'rss-channels gozerbot')
tests.add('rss-channels')

def handle_rssaddchannel(bot, ievent):

    """ 
        rss-addchannel <name> [<botname>] <channel> .. add a channel to \
        rss item.
    """

    try:
        (name, botname, channel) = ievent.args
    except ValueError:

        try:
            (name, channel) = ievent.args
            botname = bot.name
        except ValueError:

            try:
                name = ievent.args[0]
                botname = bot.name
                channel = ievent.channel
            except IndexError:
                ievent.missing('<name> [<botname>] <channel>')
                return

    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss object" % name)
        return

    if not rssitem.data.running:
        ievent.reply('%s watcher is not running' % name)
        return

    if jsonstring([botname, channel]) in rssitem.data.watchchannels or [botname, channel] in rssitem.data.watchchannels:
        ievent.reply('we are already monitoring %s on (%s,%s)' % \
(name, botname, channel))
        return

    rssitem.data.watchchannels.append([botname, channel])
    watcher.save()
    ievent.reply('%s added to %s rss item' % (channel, name))

cmnds.add('rss-addchannel', handle_rssaddchannel, ['OPER', ])
examples.add('rss-addchannel', 'rss-addchannel <name> [<botname>] <channel> \
..add <channel> or <botname> <channel> to watchchannels of <name>', \
'1) rss-addchannel gozerbot #dunkbots 2) rss-addchannel gozerbot main \
#dunkbots')
tests.add('rss-addchannel dev exec')

def handle_rsssetitems(bot, ievent):

    """ set items of a rss feed. """

    try:
        (name, items) = ievent.args[0], ievent.args[1:]
    except ValueError:
        ievent.missing('<name> <items>')
        return

    target = ievent.channel

    if not watcher.byname(name):
        ievent.reply("we don't have a %s feed" % name)
        return

    watcher.itemslists.data[jsonstring([name, target])] = items
    watcher.itemslists.save()
    ievent.reply('%s added to (%s,%s) itemslist' % (items, name, target))

cmnds.add('rss-setitems', handle_rsssetitems, ['RSS', 'USER'])
examples.add('rss-setitems', 'set tokens of the itemslist (per user/channel)',\
 'rss-setitems gozerbot author author link pubDate')
tests.add('rss-setitems dev link author')

def handle_rssadditem(bot, ievent):

    """ add an item (token) to a feeds itemslist. """

    try:
        (name, item) = ievent.args
    except ValueError:
        ievent.missing('<name> <item>')
        return

    target = ievent.channel

    if not watcher.byname(name):
        ievent.reply("we don't have a %s feed" % name)
        return

    try:
        watcher.itemslists.data[jsonstring([name, target])].append(item)
    except KeyError:
        watcher.itemslists.data[jsonstring([name, target])] = [item, ]

    watcher.itemslists.save()
    ievent.reply('%s added to (%s,%s) itemslist' % (item, name, target))

cmnds.add('rss-additem', handle_rssadditem, ['RSS', 'USER'])
examples.add('rss-additem', 'add a token to the itemslist (per user/channel)',\
 'rss-additem gozerbot link')
tests.add('rss-additem dev link')

def handle_rssdelitem(bot, ievent):

    """ delete item from a feeds itemlist. """

    try:
        (name, item) = ievent.args
    except ValueError:
        ievent.missing('<name> <item>')
        return

    target = ievent.channel

    if not watcher.byname(name):
        ievent.reply("we don't have a %s feed" % name)
        return

    try:
        watcher.itemslists.data[jsonstring([name, target])].remove(item)
        watcher.itemslists.save()
    except (RssNoSuchItem, ValueError):
        ievent.reply("we don't have a %s rss feed" % name)
        return

    ievent.reply('%s removed from (%s,%s) itemslist' % (item, name, target))

cmnds.add('rss-delitem', handle_rssdelitem, ['RSS', 'USER'])
examples.add('rss-delitem', 'remove a token from the itemslist \
(per user/channel)', 'rss-delitem gozerbot link')
tests.add('rss-delitem')

def handle_rssmarkuplist(bot, ievent):

    """ show possible markups that can be used. """

    ievent.reply('possible markups ==> ' , possiblemarkup, dot=True)

cmnds.add('rss-markuplist', handle_rssmarkuplist, ['USER', ])
examples.add('rss-markuplist', 'show possible markup entries', \
'rss-markuplist')
tests.add('rss-markuplist')

def handle_rssmarkup(bot, ievent):

    """ show the markup of a feed. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    target = ievent.channel

    try:
        ievent.reply(str(watcher.markup[jsonstring([name, target])]))
    except KeyError:
        pass

cmnds.add('rss-markup', handle_rssmarkup, ['RSS', 'USER'])
examples.add('rss-markup', 'show markup list for a feed (per user/channel)', \
'rss-markup gozerbot')
tests.add('rss-markup dev')

def handle_rssaddmarkup(bot, ievent):

    """ add a markup to a feeds markuplist. """

    try:
        (name, item, value) = ievent.args
    except ValueError:
        ievent.missing('<name> <item> <value>')
        return

    target = ievent.channel

    try:
        value = int(value)
    except ValueError:
        pass

    try:
        watcher.markup.set(jsonstring([name, target]), item, value)
        watcher.markup.save()
        ievent.reply('%s added to (%s,%s) markuplist' % (item, name, target))
    except KeyError:
        ievent.reply("no (%s,%s) feed available" % (name, target))

cmnds.add('rss-addmarkup', handle_rssaddmarkup, ['RSS', 'USER'])
examples.add('rss-addmarkup', 'add a markup option to the markuplist \
(per user/channel)', 'rss-addmarkup gozerbot all-lines 1')
tests.add('rss-addmarkup dev')

def handle_rssdelmarkup(bot, ievent):

    """ delete markup from a feeds markuplist. """

    try:
        (name, item) = ievent.args
    except ValueError:
        ievent.missing('<name> <item>')
        return

    target = ievent.channel

    try:
        del watcher.markup[jsonstring([name, target])][item]
    except (KeyError, TypeError):
        ievent.reply("can't remove %s from %s feed's markup" %  (item, name))
        return

    watcher.markup.save()
    ievent.reply('%s removed from (%s,%s) markuplist' % (item, name, target))

cmnds.add('rss-delmarkup', handle_rssdelmarkup, ['RSS', 'USER'])
examples.add('rss-delmarkup', 'remove a markup option from the markuplist \
(per user/channel)', 'rss-delmarkup gozerbot all-lines')
tests.add('rss-delmarkup dev all-lines')

def handle_rssdelchannel(bot, ievent):

    """ 
        rss-delchannel <name> [<botname>] <channel> .. delete channel \
        from rss item.

    """

    botname = None

    try:
        (name, botname, channel) = ievent.args
    except ValueError:

        try:
            (name, channel) = ievent.args
            botname = bot.name
        except ValueError:

            try:
                name = ievent.args[0]
                botname = bot.name
                channel = ievent.channel
            except IndexError:
                ievent.missing('<name> [<botname>] [<channel>]')
                return

    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss object" % name)
        return

    if jsonstring([botname, channel]) in rssitem.data.watchchannels:
        rssitem.data.watchchannels.remove(jsonstring([botname, channel]))
        ievent.reply('%s removed from %s rss item' % (channel, name))
    elif [botname, channel] in rssitem.data.watchchannels:
        rssitem.data.watchchannels.remove([botname, channel])
        ievent.reply('%s removed from %s rss item' % (channel, name))
    else:
        ievent.reply('we are not monitoring %s on (%s,%s)' % (name, botname, \
channel))
        return

    rssitem.save()

cmnds.add('rss-delchannel', handle_rssdelchannel, ['OPER', ])
examples.add('rss-delchannel', 'rss-delchannel <name> [<botname>] \
[<channel>] .. delete <channel> or <botname> <channel> from watchchannels of \
<name>', '1) rss-delchannel gozerbot #dunkbots 2) rss-delchannel gozerbot \
main #dunkbots')
tests.add('rss-delchannel dev exec')

def handle_rssstopwatch(bot, ievent):

    """ rss-stopwatch <name> .. stop a watcher thread. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    rssitem = watcher.byname(name)

    if not rssitem:
        ievent.reply("there is no %s rssitem" % name)
        return

    if not watcher.stopwatch(name):
        ievent.reply("can't stop %s watcher" % name)
        return

    ievent.reply('stopped %s rss watch' % name)

cmnds.add('rss-stopwatch', handle_rssstopwatch, ['OPER', ])
examples.add('rss-stopwatch', 'rss-stopwatch <name> .. stop polling <name>', \
'rss-stopwatch gozerbot')
tests.add('rss-stopwatch makker')

def handle_rsssleeptime(bot, ievent):

    """ rss-sleeptime <name> .. get sleeptime of rss item. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return
    if not watcher.ownercheck(name, ievent.userhost):
        ievent.reply("you are not the owner of the %s feed" % name)
        return

    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss item" % name)
        return

    try:
        ievent.reply('sleeptime for %s is %s seconds' % (name, \
str(rssitem.data.sleeptime)))
    except AttributeError:
        ievent.reply("can't get sleeptime for %s" % name)

cmnds.add('rss-sleeptime', handle_rsssleeptime, 'USER')
examples.add('rss-sleeptime', 'rss-sleeptime <name> .. get sleeping time \
for <name>', 'rss-sleeptime gozerbot')
tests.add('rss-sleeptime dev')

def handle_rsssetsleeptime(bot, ievent):

    """ rss-setsleeptime <name> <seconds> .. set sleeptime of rss item. """

    try:
        (name, sec) = ievent.args
        sec = int(sec)
    except ValueError:
        ievent.missing('<name> <seconds>')
        return

    if not watcher.ownercheck(name, ievent.userhost):
        ievent.reply("you are not the owner of the %s feed" % name)
        return

    if sec < 60:
        ievent.reply('min is 60 seconds')
        return

    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss item" % name)
        return

    rssitem.data.sleeptime = sec

    if rssitem.data.running:

        try:
            watcher.changeinterval(name, sec)
        except KeyError, ex:
            ievent.reply("failed to set interval: %s" % str(ex))
            return

    watcher.save()
    ievent.reply('sleeptime set')

cmnds.add('rss-setsleeptime', handle_rsssetsleeptime, ['USER', ])
examples.add('rss-setsleeptime', 'rss-setsleeptime <name> <seconds> .. set \
sleeping time for <name> .. min 60 sec', 'rss-setsleeptime gozerbot 600')
tests.add('rss-setsleeptime dev 60')

def handle_rssget(bot, ievent):

    """ rss-get  <name> .. fetch rss data. """
 
    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    channel = ievent.channel
    rssitem = watcher.byname(name)

    if rssitem == None:
        ievent.reply("we don't have a %s rss item" % name)
        return

    try:
        result = watcher.getdata(name)
    except Exception, ex:
        ievent.reply('%s error: %s' % (name, str(ex)))
        return

    if watcher.markup.get(jsonstring([name, channel]), 'reverse-order'):
        result = result[::-1]

    response = None
    go = watcher.markup.get(jsonstring([name, channel]), 'all-lines')

    if go:

        for i in result:
            response = watcher.makeresponse(name, [i, ], channel)
            bot.say(channel, "%s: %s" % (name, response))

    else:
        response = watcher.makeresponse(name, result, ievent.channel)

        if response:
            ievent.reply("results of %s: %s" % (name, response))
        else:
            ievent.reply("can't make a reponse out of %s" % name)

cmnds.add('rss-get', handle_rssget, ['RSS', 'USER'], threaded=True)
examples.add('rss-get', 'rss-get <name> .. get data from <name>', 'rss-get gozerbot')
tests.add('rss-get dev')

def handle_rssrunning(bot, ievent):

    """ rss-running .. show which watchers are running. """

    result = watcher.runners()
    resultlist = []
    teller = 1

    for i in result:
        resultlist.append("%s %s" % (i[0], i[1]))

    if resultlist:
        ievent.reply("running rss watchers: ", resultlist, nr=1)
    else:
        ievent.reply('nothing running yet')

cmnds.add('rss-running', handle_rssrunning, ['RSS', 'USER'])
examples.add('rss-running', 'rss-running .. get running rsswatchers', \
'rss-running')
tests.add('rss-running')

def handle_rsslist(bot, ievent):

    """ rss-list .. return list of rss items. """

    result = watcher.list()
    result.sort()

    if result:
        ievent.reply("rss items: ", result, dot=True)
    else:
        ievent.reply('no rss items yet')

cmnds.add('rss-list', handle_rsslist, ['RSS', 'USER'])
examples.add('rss-list', 'get list of rss items', 'rss-list')
tests.add('rss-list')

def handle_rssurl(bot, ievent):

    """ rss-url <name> .. return url of rss item. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    if not watcher.ownercheck(name, ievent.userhost):
        ievent.reply("you are not the owner of the %s feed" % name)
        return

    result = watcher.url(name)

    if not result:
        ievent.reply("can't fetch url for %s" % name)
        return

    try:
        if ':' in result.split('/')[1]:
            if not ievent.msg:
                ievent.reply('run this command in a private message')
                return
    except (TypeError, ValueError, IndexError):
        pass

    ievent.reply('url of %s: %s' % (name, result))

cmnds.add('rss-url', handle_rssurl, ['OPER', ])
examples.add('rss-url', 'rss-url <name> .. get url from rssitem with <name>', 'rss-url gozerbot')
tests.add('rss-url dev')

def handle_rssseturl(bot, ievent):

    """ rss-seturl <name> <url> .. set url of rss item. """

    try:
        name = ievent.args[0]
        url = ievent.args[1]
    except IndexError:
        ievent.missing('<name> <url>')
        return

    if not watcher.ownercheck(name, ievent.userhost):
        ievent.reply("you are not the owner of the %s feed" % name)
        return

    oldurl = watcher.url(name)

    if not oldurl:
        ievent.reply("no %s rss item found" % name)
        return

    if watcher.seturl(name, url):
        watcher.sync(name)
        ievent.reply('url of %s changed' % name)
    else:
        ievent.reply('failed to set url of %s to %s' % (name, url))

cmnds.add('rss-seturl', handle_rssseturl, ['USER', ])
examples.add('rss-seturl', 'rss-seturl <name> <url> .. change url of rssitem',\
'rss-seturl gozerbot http://core.gozerbot.org/hg/dev/0.9/rss-log')
tests.add('rss-seturl dev http://core.gozerbot.org/hg/dev/0.9/rss-list')

def handle_rssitemslist(bot, ievent):

    """ rss-itemslist <name> .. show itemslist of rss item. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    try:
        itemslist = watcher.itemslists[jsonstring([name, ievent.channel.lower()])]
    except KeyError:
        ievent.reply("no itemslist set for (%s, %s)" % (name, ievent.channel.lower()))
        return

    ievent.reply("itemslist of (%s, %s): " % (name, ievent.channel.lower()), itemslist)

cmnds.add('rss-itemslist', handle_rssitemslist, ['RSS', 'USER'])
examples.add('rss-itemslist', 'rss-itemslist <name> .. get itemslist of \
<name> ', 'rss-itemslist gozerbot')
tests.add('rss-itemslist dev')

def handle_rssscan(bot, ievent):

    """ rss-scan <name> .. scan rss item for used xml items. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    if not watcher.byname(name):
        ievent.reply('no %s feeds available' % name)
        return

    try:
        result = watcher.scan(name)
    except Exception, ex:
        ievent.reply(str(ex))
        return

    if result == None:
        ievent.reply("can't get data for %s" % name)
        return

    res = []

    for i in result:
        res.append("%s=%s" % i)

    ievent.reply("tokens of %s: " % name, res)

cmnds.add('rss-scan', handle_rssscan, ['USER', ])
examples.add('rss-scan', 'rss-scan <name> .. get possible items of <name> ', 'rss-scan gozerbot')
tests.add('rss-scan dev')

def handle_rsssync(bot, ievent):

    """ rss-sync <name> .. sync rss item data. """

    try:
        name = ievent.args[0]
    except IndexError:
        ievent.missing('<name>')
        return

    #if not watcher.ownercheck(name, ievent.userhost):
    #    ievent.reply("you are not the owner of the %s feed" % name)
    #    return

    try:
        result = watcher.sync(name)
        ievent.reply('%s synced' % name)
    except Exception, ex:
        ievent.reply("%s error: %s" % (name, str(ex)))

cmnds.add('rss-sync', handle_rsssync, ['USER', ])
examples.add('rss-sync', 'rss-sync <name> .. sync data of <name>', \
'rss-sync gozerbot')
tests.add('rss-sync dev')

def handle_rssfeeds(bot, ievent):

    """ rss-feeds <channel> .. show what feeds are running in a channel. """

    try:
        channel = ievent.args[0]
    except IndexError:
        channel = ievent.channel

    try:
        result = watcher.getfeeds(bot.name, channel)

        if result:
            ievent.reply("feeds running in %s: " % channel, result, dot=True)
        else:
            ievent.reply('%s has no feeds running' % channel)

    except Exception, ex:
        ievent.reply("ERROR: %s" % str(ex))

cmnds.add('rss-feeds', handle_rssfeeds, ['USER', 'RSS'])
examples.add('rss-feeds', 'rss-feeds <name> .. show what feeds are running \
in a channel', '1) rss-feeds 2) rss-feeds #dunkbots')
tests.add('rss-feeds')

def handle_rsslink(bot, ievent):

    """ search link entries in cached data. """

    try:
        feed, rest = ievent.rest.split(' ', 1)
    except ValueError:
        ievent.missing('<feed> <words to search>')
        return

    rest = rest.strip().lower()

    try:
        res = watcher.search(feed, 'link', rest)

        if not res:
            res = watcher.search(feed, 'feedburner:origLink', rest)

        if res:
            ievent.reply("link: ", res, dot=" \002||\002 ")

    except KeyError:
        ievent.reply('no %s feed data available' % feed)
        return

cmnds.add('rss-link', handle_rsslink, ['RSS', 'USER'])
examples.add('rss-link', 'give link of item which title matches search key', \
'rss-link gozerbot gozer')
tests.add('rss-link dev gozerbot')

def handle_rssdescription(bot, ievent):

    """ search descriptions in cached data. """

    try:
        feed, rest = ievent.rest.split(' ', 1)
    except ValueError:
        ievent.missing('<feed> <words to search>')
        return

    rest = rest.strip().lower()
    res = ""

    try:
        ievent.reply("results: ", watcher.search(feed, 'description', rest))
    except KeyError:
        ievent.reply('no %s feed data available' % feed)
        return

cmnds.add('rss-description', handle_rssdescription, ['RSS', 'USER'])
examples.add('rss-description', 'give description of item which title \
matches search key', 'rss-description gozerbot gozer')
tests.add('rss-description')

def handle_rssall(bot, ievent):

    """ search titles of all cached data. """

    try:
        feed = ievent.args[0]
    except IndexError:
        ievent.missing('<feed>')
        return

    try:
        ievent.reply('results: ', watcher.all(feed, 'title'), dot=" \002||\002 ")
    except KeyError:
        ievent.reply('no %s feed data available' % feed)
        return

cmnds.add('rss-all', handle_rssall, ['RSS', 'USER'])
examples.add('rss-all', "give titles of a feed", 'rss-all gozerbot')
tests.add('rss-all dev')

def handle_rsssearch(bot, ievent):

    """ search in titles of cached data. """

    try:
        txt = ievent.args[0]
    except IndexError:
        ievent.missing('<txt>')
        return

    try:        
        ievent.reply("results: ", watcher.searchall('title', txt))
    except KeyError:
        ievent.reply('no %s feed data available' % feed)
        return

cmnds.add('rss-search', handle_rsssearch, ['RSS', 'USER'])
examples.add('rss-search', "search titles of all current feeds", \
'rss-search goz')
tests.add('rss-search gozerbot')

def handle_rsspeek(bot, ievent):

    """ run a peek of a feed. """

    if not ievent.rest:
        ievent.missing('<feedname>')
        return

    if not watcher.peek(ievent.rest, event=ievent):
        ievent.reply("no new entries for feed %s" % ievent.rest)
    else:
        ievent.done()

cmnds.add('rss-peek', handle_rsspeek, ['USER', ])
tests.add('rss-peek dev')

def handle_rsspeekall(bot, ievent):

    """ peek all feeds. """

    watcher.peekall(ievent.rest, event=ievent)
    ievent.done()

cmnds.add('rss-peekall', handle_rsspeekall, ['OPER', ])
tests.add('rss-peek dev')

